package exp.web.cnn;

import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import exp.URLContent;

import tools.FileReaderEnhance;
import tools.twitter.TweetTidy;

/**
 * 提取CNN的highlights和内容
 * 
 * @author Lanjun
 * 
 */
public class CNNExtraction {
	// CNN highlights
	static final Pattern CNNHL = Pattern.compile("<li>(.*?)(</li>|<br />)");
	// CNN contents
	static final Pattern CNNC = Pattern
			.compile(
					"<ul class=\"cnn_bulletbin cnnStryHghLght\">.*</p>\\s*<!--endclickprintinclude-->",
					Pattern.DOTALL);
	// CNN contents
	static final Pattern CNNP = Pattern.compile("<p.*?>(.*?)</p>");
	// CNN Title
	static final Pattern CNNT = Pattern.compile("<title>(.*?)</title>");
	//highlights tidy patterns
	static final Pattern [] hlPatterns = {
		Pattern.compile("<b>\\s*NEW:\\s*</b>", Pattern.CASE_INSENSITIVE),
		Pattern.compile("</?[^>]+>"),
		Pattern.compile("Share your ideas"),
	};
	//content tidy patterns
	static final Pattern [] cPatterns = {
		Pattern.compile("</?[^>]+>"),
		Pattern.compile("\\(CNN\\) -- "),
	};

	public static void main(String[] args) {
		URLContent cnn1 = getCNN(FileReaderEnhance.readToString("data/test/cnn1.htm"), "http://testurl1");
		System.out.println("TITLE:" + cnn1.title);
		for (String p : cnn1.contents) {
			System.out.println(p);
		}
		System.out.println();
		URLContent cnn2 = getCNN(FileReaderEnhance.readToString("data/test/cnn2.htm"), "http://testurl2");
		System.out.println("TITLE:" + cnn2.title);
		for (String p : cnn2.contents) {
			System.out.println(p);
		}
		System.out.println();
		URLContent cnn3 = getCNN(FileReaderEnhance.readToString("data/test/cnn3.htm"), "http://testurl3");
		System.out.println("TITLE:" + cnn3.title);
		for (String p : cnn3.contents) {
			System.out.println(p);
		}
		System.out.println();
		URLContent cnn4 = getCNN(FileReaderEnhance.readToString("data/test/cnn4.htm"), "http://testurl3");
		System.out.println("TITLE:" + cnn4.title);
		for (String p : cnn4.contents) {
			System.out.println(p);
		}
		System.out.println();
		URLContent cnn5 = getCNN(FileReaderEnhance.readToString("data/test/cnn5.htm"), "http://testurl3");
		System.out.println("TITLE:" + cnn5.title);
		for (String p : cnn5.contents) {
			System.out.println(p);
		}
		System.out.println();
		URLContent cnn6 = getCNN(FileReaderEnhance.readToString("data/test/cnn6.htm"), "http://testurl3");
		System.out.println("TITLE:" + cnn6.title);		
		System.out.println();
		URLContent cnn7 = getCNN(FileReaderEnhance.readToString("data/test/cnn7.htm"), "http://testurl3");
		System.out.println("TITLE:" + cnn7.title);
		for (String p : cnn7.contents) {
			System.out.println(p);
		}
	}

	public static URLContent getCNN(String htmlContent, String url) {
		URLContent hc = new URLContent();
		hc.url = url;
		if (htmlContent == null) {
			return hc;
		}
		// get title
		Matcher tm = CNNT.matcher(htmlContent);
		while (tm.find()) {
			String title = tm.group(1);
			hc.title = title;
		}

		int hStart = htmlContent.indexOf("<ul class=\"cnn_bulletbin cnnStryHghLght\">");
		if (hStart == -1) {
			// no highlights
			hc.highlights = null;
		} else {
			hc.highlights = new ArrayList<>();
			int hEnd = htmlContent.indexOf("</ul>", hStart + 41);
			String hl = htmlContent.substring(hStart + 41, hEnd);
			Matcher m = CNNHL.matcher(hl);
			while (m.find()) {
				String oneHL = m.group(1);
				hc.highlights.add(oneHL);
			}
		}
		
		// get the contents
		if (hc.highlights == null) {
			return hc;
		}
		String contentRegion = null;
		Matcher c = CNNC.matcher(htmlContent);
		if (c.find()) {
			contentRegion = c.group(0);
		}
		if (contentRegion == null) {
			return hc;
		}
		hc.contents = new ArrayList<>();
		Matcher p = CNNP.matcher(contentRegion);
		while (p.find()) {
			String oneP = p.group(1);
			hc.contents.add(oneP);
		}
		if (hc.contents.size() == 0) {
			System.out.println("ERROR:" + hc.url);
		}

		return hc;
	}

	public static String unify(String url) {
		int position = url.indexOf("index.html");
		if (position == -1) {
			if (url.endsWith("/")) {
				return url + "index.html";
			} else {
				int position1 = url.lastIndexOf("/");
				return url.substring(0, position1 + 1) + "index.html";
			}
		} else {
			return url.substring(0, position + 10);
		}
	}

	public static URLContent tidy(URLContent uc) {
		for (int i = 0; i < uc.highlights.size(); i++) {
			String hl = uc.highlights.get(i);
			hl = TweetTidy.doTidyHTML(hl);
			for (Pattern p : hlPatterns) {
				hl = p.matcher(hl).replaceAll("");
			}
			uc.highlights.set(i, hl);
		}
		for (int i = 0; i < uc.contents.size(); i++) {
			String c = uc.contents.get(i);
			c = TweetTidy.doTidyHTML(c);
			for (Pattern p : cPatterns) {
				c = p.matcher(c).replaceAll("");
			}
			uc.contents.set(i, c);
		}
		
		return uc;
	}
}
